import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;


public class ConvertToLibSVM {

	/**
	 * @param args
	 * @throws IOException 
	 */
	public static void main(String[] args) throws IOException {
		
		String fileVectorName = "data_giaitri_temp.txt";
		String[] vectorArray = readFile(fileVectorName).split("\n");
		
		//read and save temp vector
		List<Article> articleList = new LinkedList<Article>();
		for(int i = 0; i < vectorArray.length; i++){
//					System.out.println(vectorArray[i]);
			String[] tempVectorContent = null;
			tempVectorContent = vectorArray[i].split(" ");		
			Article tempArticle = new Article(tempVectorContent[0],"",Integer.parseInt(tempVectorContent[1]),0,0,0);
			articleList.add(tempArticle);
		}
		
		File f = new File("datatrain_giaitri.txt");
	    FileWriter fw = new FileWriter(f, true);
	    BufferedWriter bw = new BufferedWriter(fw);		
	    String line;
		
		//read document
		String fileWithoutPuctuation= "";
		int numOfArticle = 100;
		
		List<Article> articleVector;
		
		for(int i = 0; i < numOfArticle; i++){
			
			String fileName = "";
			fileName = "giaitri (" + (i+1) + ").txt";
//			if((i + 1) < 10){
//				fileName = "vnexpress000" + (i+1) + ".doisong";
//			} else if((i + 1)/10 >= 1 && (i + 1)/10 < 10){
//				fileName = "vnexpress00" + (i+1) + ".doisong";
//			} else if((i + 1)/100 >= 1 && (i + 1)/100 < 10){
//				fileName = "vnexpress0" + (i+1) + ".doisong";
//			} else if((i + 1)/1000 >= 1 && (i + 1)/1000 < 10){
//				fileName = "vnexpress" + (i+1) + ".doisong";
//			}
			
			fileWithoutPuctuation = readFile(fileName);
			//remove special characters
			fileWithoutPuctuation = fileWithoutPuctuation.replaceAll("[^\\p{L}\\p{Nd}]"," ");
			//---------------------------------------
			String[] articleWords = fileWithoutPuctuation.split(" ");
			
			HashMap<String, Integer> hashMap = new HashMap<String, Integer>();
			
			//save word hashmap which contains unique words of document and its frequency 
			for(int j = 0; j < articleWords.length; j++){
				Integer freq = hashMap.get(articleWords[j]);
				if(freq == null){
					freq = 1;
				} else {
					freq++;
				}
				// max length of word is 7 in Vietnamese
				if(!articleWords[j].matches("[0-9]+") && !articleWords[j].equals("")){
					hashMap.put(articleWords[j], freq);
				} 				
			}
			
			//calculate length of document after standardize
			int docSize = 0;
			for(int k = 0; k < articleList.size(); k++){
				if(hashMap.containsKey(articleList.get(k).getWord())){
					docSize += hashMap.get(articleList.get(k).getWord());
				}
			}
			
//			System.out.println(docSize);
			String word;
			int numOfWord;
			articleVector = new LinkedList<Article>();
			for(int k = 0; k < articleList.size(); k++){
				word = articleList.get(k).getWord();
				numOfWord = articleList.get(k).getNumOfWord(); //number of documents which contains this word
				if(hashMap.containsKey(word)){
					//if this word exist in this document, do: 
					if(hashMap.get(word) > 0){
						double tf = 0;
						double idf = 0;
						double tfidf = 0;
						double decimal = Math.pow(10, 8);
						Article tempArticle = new Article();
//						tempArticle.setWord(word);
//						tempArticle.setNumOfWord(articleList.get(k).getNumOfWord());
//						tempArticle.setNumOfWordADoc(hashMap.get(word));
//						tempArticle.setDocLength(articleList.size());
						if(numOfWord > 0){
							tf 	= (double)hashMap.get(word)/docSize; 
							idf = (double)Math.log10(numOfArticle/numOfWord);
							tfidf = Math.round(tf*idf*decimal)/decimal;
							if(tfidf > 0) {
								tempArticle.setTfidf(BigDecimal.valueOf(tfidf).toPlainString());
							}
							
//							tempArticle.setIfidf(tfidf);
//							System.out.println("tf tuso  :  " + tempArticle.getNumOfWordADoc());
//							System.out.println("docSize  :  " + docSize);
//							System.out.println("tf  :  " + tf);
//							System.out.println("idf :  " + idf);
//							System.out.printf(tempArticle.getTfidf());
//							System.out.println();
							articleVector.add(tempArticle);
						} else {
							tempArticle = new Article(articleList.get(k).getWord(),"",articleList.get(k).getNumOfWord(),0,0,0);
							articleVector.add(tempArticle);
						}						
					}					
				} else {
					Article tempArticle = new Article(articleList.get(k).getWord(),"",articleList.get(k).getNumOfWord(),0,0,0);
					articleVector.add(tempArticle);
				}
			}
			
//			System.out.println(hashMap);
			line = "" + 2 + " ";
			for(int k = 0; k < articleVector.size(); k++){
//				System.out.println(articleVector.get(k).getWord());
//				System.out.println(articleVector.get(k).getNumOfWord());
//				System.out.println(articleVector.get(k).getNumOfWordADoc());
//				System.out.println(articleVector.get(k).getDocLength());
//				System.out.println(articleVector.get(k).getIfidf());
//				System.out.println("-------------------------------");
				
				if(articleVector.get(k).getTfidf() != ""){
					line += (k+1) + ":" + articleVector.get(k).getTfidf() + " ";
				}
			}
			System.out.println((i+1) + " " + articleList.size() + " " + articleVector.size() + " " + hashMap.size() + " " + docSize + " " + line);
			bw.write(line);
		    bw.newLine();
		    
		    hashMap.clear();
		    articleVector.clear();
		}		
		bw.close();
		
		
		
//		for(int i = 0; i < articleList.size(); i++){
//			System.out.println(articleList.get(i).getWord());
//			System.out.println(articleList.get(i).getNumOfWord());
//			System.out.println(articleList.get(i).getNumOfWordADoc());
//			System.out.println(articleList.get(i).getDocLength());
//			System.out.println(articleList.get(i).getIfidf());
//			System.out.println("-------------------------------");
//		}

	}
	
	static String readFile(String fileName) {
		String temp = "";
		try (BufferedReader br = new BufferedReader(new FileReader(fileName))) {
			String sCurrentLine;
			while ((sCurrentLine = br.readLine()) != null) {
				temp += sCurrentLine + "\n";
			}
		} catch (IOException e) {
			e.printStackTrace();
		}

		return temp;
	}

	static void writeFile(String fileName, String writeString) {
		try {
			File fileDir = new File(fileName);
			BufferedWriter out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileDir), "UTF8"));
			
			out.append(writeString);
			
			out.flush();
			out.close();
		} catch (UnsupportedEncodingException e) {
			System.out.println(e.getMessage());
		} catch (IOException e) {
			System.out.println(e.getMessage());
		} catch (Exception e) {
			System.out.println(e.getMessage());
		}
	}
}
